﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;

namespace Utils
{
    public class TFIDFSol
    {
        public TFIDFSol()
        {
            db = new List<Document>();
            
            dbKeywords = new List<String>();

            idfs = new Dictionary<String, Double>();

            invertedFile = new Dictionary<String, List<int>>();

            tf = new List<Dictionary<String, Double>>();
        }
        public List<Document> db { get; set; } // the document collection
        public List<String> dbKeywords { get; set; } // the document collection
        public Dictionary<String, Double> idfs { get; set; } // idf value for each term in the vocabulary
        public Dictionary<String, List<int>> invertedFile { get; set; } // term -> docIds of docs containing the term
        public List<Dictionary<String, Double>> tf { get; set; } // term x docId matrix with term frequencies
        public List<Group> groups { get; set; }

        public void Group(int k, int maxIteration)
        {
            //1. init documents collection

            // init global variables: tf, invertedFile, and idfs
            init();
            //DocScore[,] similarityMatrix = new DocScore[db.Count, db.Count];
            //foreach (Document doc in db)
            //{
            //    foreach (Document docIn in db)
            //    {
            //        similarityMatrix[doc.Id, docIn.Id] = new DocScore(similarity(getDocVec(doc.Id), getDocVec(docIn.Id)), doc.Id);
            //    }
            //}
            groups = new List<Group>();
            List<int> centroidsIds = new List<int>();
            List<Document> beginigCentroids = new List<Document>();
            Random r = new Random();
            int rInt = 0;
            for (int i = 0; i < k; i++)
            {
                do
                {
                    rInt = r.Next(0, db.Count - 1); //for ints
                }
                while (centroidsIds.Contains(rInt));
                groups.Add(new Group { CentroidVector = getDocVec(rInt), DocsAndSimToCentr = new Dictionary<int, double> { { rInt, 1 } } });
                centroidsIds.Add(rInt);
                beginigCentroids.Add(db.FirstOrDefault(x => x.Id == rInt));
            }

            AssignToGroups(groups);
            List<Group> oldGroups = new List<Group>();
            List<List<Group>> iterations = new List<List<Group>>();
            List<Double> JValues = new List<double>();
            for (int i = 1; i < maxIteration; i++)
            {
                JValues.Add(CalculateJ(groups));
                oldGroups = groups;
                foreach (Group g in groups)
                {
                    g.CentroidVector = CalculateNewCentroid(g);
                    g.DocsAndSimToCentr = new Dictionary<int, double>();
                }

                AssignToGroups(groups);
                if (CheckGroupsConstancy(groups, oldGroups))
                    break;
            }
        }

        private bool CheckGroupsConstancy(List<Group> newGroup, List<Group> oldGroup)
        {
            for (int i = 0; i < newGroup.Count; i++)
            {
                foreach (int docId in newGroup[i].DocsAndSimToCentr.Keys)
                {
                    if (!oldGroup[i].DocsAndSimToCentr.Keys.Contains(docId))
                        return false;
                }
            }
            return true;
        }

        private void AssignToGroups(List<Group> centroids)
        {
            foreach (Document doc in db)
            {
                if (centroids.FirstOrDefault(x => x.DocsAndSimToCentr.ContainsKey(doc.Id)) != null)
                    continue;
                double max = -1;
                Group maxCentroid = null;
                foreach (Group group in centroids)
                {
                    double score = similarity(group.CentroidVector, getDocVec(doc.Id));
                    if (score > max)
                    {
                        max = score;
                        maxCentroid = group;
                    }
                }
                maxCentroid.DocsAndSimToCentr.Add(doc.Id, max);
            }
        }

        private double CalculateJ(List<Group> centroids)
        {
            double result = 0;
            foreach (Group group in centroids)
            {
                result += group.DocsAndSimToCentr.Values.Sum();
            }
            return result;
        }

        private Dictionary<string, double> CalculateNewCentroid(Group group)
        {
            Dictionary<string, double> sums = new Dictionary<string, double>();
            foreach (KeyValuePair<int, double> doc in group.DocsAndSimToCentr)
            {
                Dictionary<string, double> docVec = getDocVec(doc.Key);
                foreach (KeyValuePair<string, double> termVal in docVec)
                {
                    if (!sums.ContainsKey(termVal.Key))
                    {
                        sums.Add(termVal.Key, termVal.Value);
                    }
                    else
                    {
                        sums[termVal.Key] += termVal.Value;
                    }
                }
            }
            Dictionary<string, double> result = new Dictionary<string, double>();
            foreach (KeyValuePair<string, double> termVal in sums)
            {
                result[termVal.Key] = termVal.Value / group.DocsAndSimToCentr.Count;
            }
            return result;
        }


        public void Process(string query)
        {

            // init global variables: tf, invertedFile, and idfs
            init();
            DocScore[,] scores = new DocScore[db.Count, db.Count];
            foreach (Document doc in db)
            {
                foreach (Document docIn in db)
                {
                    scores[doc.Id, docIn.Id] = new DocScore(similarity(getDocVec(doc.Id), getDocVec(docIn.Id)), doc.Id);
                }
            }

            // similarities for different queries
            rank(query);
        }


        // calculates the similarity between two vectors
        // each vector is a term -> weight map
        private double similarity(Dictionary<String, Double> v1, Dictionary<String, Double> v2)
        {
            double sum = 0;
            // iterate through one vector
            foreach (KeyValuePair<String, Double> entry in v1)
            {
                String term = entry.Key;
                Double w1 = entry.Value;
                // multiply weights if contained in second vector
                Double? w2 = v2.ContainsKey(term) ? v2[term] : (Double?)null;
                if (w2.HasValue)
                    sum += w1 * w2.Value;
            }
            double result = sum / (vecLength(v1.Values.ToList()) * vecLength(v2.Values.ToList()));
            return result;
        }

        // returns the length of a vector
        private double vecLength(List<Double> vec)
        {
            double sum = 0;
            foreach (Double d in vec)
            {
                sum += Math.Pow(d, 2);
            }
            return Math.Sqrt(sum);
        }

        // ranks a query to the documents of the database
        private void rank(String query)
        {
            // get term frequencies for the query terms
            Dictionary<String, Double> termFreqs = getTF(query, true);

            // construct the query vector
            // the query vector
            Dictionary<String, Double> queryVec = new Dictionary<String, Double>();

            // iterate through all query terms
            foreach (KeyValuePair<String, Double> entry in termFreqs)
            {
                String term = entry.Key;
                //DONE//TODO compute tfidf value for terms of query
                double tfidf = idf(term) * entry.Value;
                queryVec[term] = tfidf;
            }




            List<int> union = new List<int>();
            List<String> queryTerms = new List<String>(termFreqs.Where(x => x.Value > 0).Select(x => x.Key));
            if (queryTerms.Count > 0)
            {
                // from the inverted file get the union of all docIDs that contain any query term
                union = invertedFile[queryTerms.First()];
                foreach (String term in queryTerms)
                {
                    foreach (int di in invertedFile[term])
                    {
                        if (!union.Contains(di))
                            union.Add(di);
                    }
                }
            }
            // calculate the scores of documents in the union
            List<DocScore> scores = new List<DocScore>();
            foreach (int i in union)
            {
                scores.Add(new DocScore(similarity(queryVec, getDocVec(i)), i));
            }

            // sort and print the scores
            //scores = scores.OrderByDescending(x => x.score).ToList();

            foreach (Document dc in db)
            {
                DocScore d = scores.FirstOrDefault(x => x.docId == dc.Id);
                if(d != null)
                {
                    dc.Score = d.score;
                }
                else
                {
                    dc.Score = 0;
                }
            }

            db = db.OrderByDescending(x => x.Score).ThenBy(x => x.Id).ToList();
        }
    

        // returns the idf of a term
        private double idf(String term)
        {
            return idfs[term];
        }

        // calculates the document vector for a given docID
        private Dictionary<String, Double> getDocVec(int docId)
        {
            Dictionary<String, Double> vec = new Dictionary<String, Double>();

            // get all term frequencies
            Dictionary<String, Double> termFreqs = tf[docId];

            // for each term, tf * idf
            foreach (KeyValuePair<String, Double> entry in termFreqs)
            {
                String term = entry.Key;
                double tfidf = idf(term) * entry.Value;
                vec[term] = tfidf;
            }
            return vec;
        }

        // calculates the term frequencies for a document
        private Dictionary<string, double> getTF(string doc, bool checkQuery)
        {
            Dictionary<String, Double> termFreqs = new Dictionary<String, Double>();
            double max = 0;


            // for all tokens
            foreach (string term in dbKeywords)//doc.Split(' ')) 
            {
                Double count = 0;
                foreach (string word in doc.Split(' '))
                {
                    if (word.Contains(term))
                        count++;
                }
                Double readCount = 0;
                // count the max term frequency
                if (termFreqs.ContainsKey(term))
                {
                    readCount = termFreqs[term];
                }
                else
                {
                    termFreqs.Add(term, 0);
                }
                readCount += count;
                termFreqs[term] = readCount;
                if (readCount > max) max = readCount;
            }

            Dictionary<String, Double> termFreqsResult = new Dictionary<String, Double>(termFreqs);
            // normalize tf
            if (max > 0)
            {
                foreach (KeyValuePair<string, Double> tf in termFreqs)
                {
                    //DONE, TODO write the formula for normalization of TF
                    termFreqsResult[tf.Key] = tf.Value / max;
                }
            }
            return termFreqsResult;
        }

        // init tf, invertedFile, and idfs
        private void init()
        {
            int docId = 0;
            // for all docs in the database
            foreach (Document d in db)
            {
                string doc = d.TitleStemmed + " " + d.ContentStemmed;
                // get the tfs for a doc
                Dictionary<string, Double> termFreqs = getTF(doc, false);

                // add to global tf vector
                tf.Add(termFreqs);

                // for all terms
                foreach (KeyValuePair<String, Double> term in termFreqs)
                {
                    if (term.Value > 0)
                    {
                        // add the current docID to the posting list
                        List<int> docIds = new List<int>();
                        if (invertedFile.ContainsKey(term.Key)) docIds = invertedFile[term.Key];
                        docIds.Add(docId);
                        invertedFile[term.Key] = docIds;
                    }
                    else if (!invertedFile.ContainsKey(term.Key))
                    {
                        invertedFile[term.Key] = new List<int>();
                    }
                }
                d.Id = docId;
                docId++;
            }

            // calculate idfs
            int dbSize = db.Count;
            // for all terms
            foreach (KeyValuePair<String, List<int>> entry in invertedFile)
            {
                String term = entry.Key;
                // get the size of the posting list, i.e. the document frequency
                int df = entry.Value.Count;


                //DONE//TODO write the formula for calculation of IDF
                //Math.log(df);
                idfs[term] = df == 0 ? 0 : Math.Log(((double)dbSize / df));
            }
        }
        // helper class to store a docId and its score
        class DocScore : IComparable<DocScore>
        {
            public double score;
            public int docId;

            public DocScore(double score, int docId)
            {
                this.score = score;
                this.docId = docId;
            }


            public int CompareTo(DocScore docScore)
            {
                if (score > docScore.score) return -1;
                if (score < docScore.score) return 1;
                return 0;
            }
        }
    }
}